/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.handler.component;

import com.google.common.hash.HashFunction;
import com.tdunning.math.stats.AVLTreeDigest;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.commons.math3.util.Combinations;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.StatsField.HllOptions;
import org.apache.solr.handler.component.StatsField.Stat;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.hll.HLL;
import org.junit.BeforeClass;

/** Statistics Component Test (which also checks some equivalent json.facet functionality) */
public class StatsComponentTest extends SolrTestCaseJ4 {

  static final String XPRE = "/response/lst[@name='stats']/";

  @BeforeClass
  public static void beforeClass() throws Exception {
    // we need DVs on point fields to compute stats & facets
    if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
      System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
    initCore("solrconfig.xml", "schema11.xml");
  }

  @Override
  public void setUp() throws Exception {
    super.setUp();
    clearIndex();
    assertU(commit());
  }

  public void testStats() throws Exception {
    for (String f :
        new String[] {
          "stats_i", "stats_l", "stats_f", "stats_d",
          "stats_ti", "stats_tl", "stats_tf", "stats_td",
          "stats_ti_dv", "stats_tl_dv", "stats_tf_dv", "stats_td_dv",
          "stats_ti_ni_dv", "stats_tl_ni_dv", "stats_tf_ni_dv", "stats_td_ni_dv",
          "stats_i_ni_p", "stats_l_ni_p", "stats_f_ni_p", "stats_d_ni_p",
        }) {

      // all of our checks should work with all of these params
      // ie: with or w/o these excluded filters, results should be the same.
      SolrParams[] baseParamsSet =
          new SolrParams[] {
            // NOTE: doTestFieldStatisticsResult needs the full list of possible tags to exclude
            params(StatsParams.STATS_FIELD, f, StatsParams.STATS, "true"),
            params(
                "stats.field",
                "{!ex=fq1,fq2}" + f,
                "stats",
                "true",
                "fq",
                "{!tag=fq1}-id_i:[0 TO 2]",
                "fq",
                "{!tag=fq2}-id_i:[2 TO 1000]"),
            params("stats.field", "{!ex=fq1}" + f, "stats", "true", "fq", "{!tag=fq1}id:1")
          };

      doTestFieldStatisticsResult(f, baseParamsSet);
      doTestFieldStatisticsMissingResult(f, baseParamsSet);
      doTestFacetStatisticsResult(f, baseParamsSet);
      doTestFacetStatisticsMissingResult(f, baseParamsSet);

      clearIndex();
      assertU(commit());
    }

    for (String f :
        new String[] {
          "stats_ii",
          "stats_tis",
          "stats_tfs",
          "stats_tls",
          "stats_tds", // trie fields
          "stats_tis_dv",
          "stats_tfs_dv",
          "stats_tls_dv",
          "stats_tds_dv", // Doc Values
          "stats_tis_ni_dv",
          "stats_tfs_ni_dv",
          "stats_tls_ni_dv",
          "stats_tds_ni_dv", // Doc Values Not indexed
          "stats_is_p",
          "stats_fs_p",
          "stats_ls_p",
          "stats_ds_p", // Point Fields
          "stats_is_ni_p",
          "stats_fs_ni_p",
          "stats_ls_ni_p" // Point Doc Values Not indexed
        }) {

      doTestMVFieldStatisticsResult(f);
      clearIndex();
      assertU(commit());
    }
  }

  public void doTestFieldStatisticsResult(String f, SolrParams[] baseParamsSet) {
    // used when doing key overrides in conjunction with the baseParamsSet
    //
    // even when these aren't included in the request, using them helps us
    // test the code path of an exclusion that refers to a fq that doesn't exist
    final String all_possible_ex = "fq1,fq2";

    assertU(adoc("id", "1", f, "-10"));
    assertU(adoc("id", "2", f, "-20"));
    assertU(commit());
    assertU(adoc("id", "3", f, "-30"));
    assertU(adoc("id", "4", f, "-40"));
    assertU(commit());

    final String fpre = XPRE + "lst[@name='stats_fields']/lst[@name='" + f + "']/";

    final String key = "key_key";
    final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='" + key + "']/";

    // status should be the same regardless of baseParams
    for (SolrParams baseParams : baseParamsSet) {
      for (String ct : new String[] {"stats.calcdistinct", "f." + f + ".stats.calcdistinct"}) {
        assertQ(
            "test statistics values using: " + ct,
            req(baseParams, "q", "*:*", ct, "true"),
            fpre + "double[@name='min'][.='-40.0']",
            fpre + "double[@name='max'][.='-10.0']",
            fpre + "double[@name='sum'][.='-100.0']",
            fpre + "long[@name='count'][.='4']",
            fpre + "long[@name='missing'][.='0']",
            fpre + "long[@name='countDistinct'][.='4']",
            "count(" + fpre + "arr[@name='distinctValues']/*)=4",
            fpre + "double[@name='sumOfSquares'][.='3000.0']",
            fpre + "double[@name='mean'][.='-25.0']",
            fpre + "double[@name='stddev'][.='12.909944487358056']");

        assertQ(
            "test statistics w/fq using: " + ct,
            req(baseParams, "q", "*:*", "fq", "-id:4", ct, "true"),
            fpre + "double[@name='min'][.='-30.0']",
            fpre + "double[@name='max'][.='-10.0']",
            fpre + "double[@name='sum'][.='-60.0']",
            fpre + "long[@name='count'][.='3']",
            fpre + "long[@name='missing'][.='0']",
            fpre + "long[@name='countDistinct'][.='3']",
            "count(" + fpre + "arr[@name='distinctValues']/*)=3",
            fpre + "double[@name='sumOfSquares'][.='1400.0']",
            fpre + "double[@name='mean'][.='-20.0']",
            fpre + "double[@name='stddev'][.='10.0']");

        // now do both in a single query

        assertQ(
            "test statistics w & w/fq via key override using: " + ct,
            req(
                baseParams,
                "q",
                "*:*",
                ct,
                "true",
                "fq",
                "{!tag=key_ex_tag}-id:4",
                "stats.field",
                "{!key=" + key + " ex=key_ex_tag," + all_possible_ex + "}" + f),

            // field name key, fq is applied
            fpre + "double[@name='min'][.='-30.0']",
            fpre + "double[@name='max'][.='-10.0']",
            fpre + "double[@name='sum'][.='-60.0']",
            fpre + "long[@name='count'][.='3']",
            fpre + "long[@name='missing'][.='0']",
            fpre + "long[@name='countDistinct'][.='3']",
            "count(" + fpre + "arr[@name='distinctValues']/*)=3",
            fpre + "double[@name='sumOfSquares'][.='1400.0']",
            fpre + "double[@name='mean'][.='-20.0']",
            fpre + "double[@name='stddev'][.='10.0']",

            // overridden key, fq is excluded
            kpre + "double[@name='min'][.='-40.0']",
            kpre + "double[@name='max'][.='-10.0']",
            kpre + "double[@name='sum'][.='-100.0']",
            kpre + "long[@name='count'][.='4']",
            kpre + "long[@name='missing'][.='0']",
            kpre + "long[@name='countDistinct'][.='4']",
            "count(" + kpre + "arr[@name='distinctValues']/*)=4",
            kpre + "double[@name='sumOfSquares'][.='3000.0']",
            kpre + "double[@name='mean'][.='-25.0']",
            kpre + "double[@name='stddev'][.='12.909944487358056']");
      }
    }

    // we should be able to compute exact same stats for a field even
    // when we specify it using the "field()" function, or use other
    // identify equivalent functions
    for (String param :
        new String[] {
          // bare
          "{!key=" + key + " ex=key_ex_tag}" + f,
          "{!key=" + key + " ex=key_ex_tag v=" + f + "}",
          // field func
          "{!lucene key=" + key + " ex=key_ex_tag}_val_:\"field(" + f + ")\"",
          "{!func key=" + key + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + key + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + key + " ex=key_ex_tag v=field(" + f + ")}",
          "{!type=func key=" + key + " ex=key_ex_tag v='field(" + f + ")'}",
          // identity math functions
          "{!type=func key=" + key + " ex=key_ex_tag v='sum(0," + f + ")'}",
          "{!type=func key=" + key + " ex=key_ex_tag v='product(1," + f + ")'}",
        }) {

      assertQ(
          "test statistics over field specified as a function: " + param,
          // NOTE: baseParams aren't used, we're looking at the function
          req(
              "q",
              "*:*",
              "stats",
              "true",
              "stats.calcdistinct",
              "true",
              "fq",
              "{!tag=key_ex_tag}-id:4",
              "stats.field",
              param),
          kpre + "double[@name='min'][.='-40.0']",
          kpre + "double[@name='max'][.='-10.0']",
          kpre + "double[@name='sum'][.='-100.0']",
          kpre + "long[@name='count'][.='4']",
          kpre + "long[@name='missing'][.='0']",
          kpre + "long[@name='countDistinct'][.='4']",
          "count(" + kpre + "arr[@name='distinctValues']/*)=4",
          kpre + "double[@name='sumOfSquares'][.='3000.0']",
          kpre + "double[@name='mean'][.='-25.0']",
          kpre + "double[@name='stddev'][.='12.909944487358056']");
    }

    // now get stats over a non-trivial function on our (single) field
    String func = "product(2, " + f + ")";
    assertQ(
        "test function statistics & key override",
        // NOTE: baseParams aren't used, we're looking at the function
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.calcdistinct",
            "true",
            "fq",
            "{!tag=key_ex_tag}-id:4",
            "stats.field",
            "{!func key=" + key + " ex=key_ex_tag}" + func),
        kpre + "double[@name='min'][.='-80.0']",
        kpre + "double[@name='max'][.='-20.0']",
        kpre + "double[@name='sum'][.='-200.0']",
        kpre + "long[@name='count'][.='4']",
        kpre + "long[@name='missing'][.='0']",
        kpre + "long[@name='countDistinct'][.='4']",
        "count(" + kpre + "arr[@name='distinctValues']/*)=4",
        kpre + "double[@name='sumOfSquares'][.='12000.0']",
        kpre + "double[@name='mean'][.='-50.0']",
        kpre + "double[@name='stddev'][.='25.81988897471611']");

    // simple cardinality over a numeric field
    assertQ(
        "test function statistics & key override",
        // NOTE: baseParams aren't used, we're looking only at the cardinality
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "fq",
            "{!tag=key_ex_tag}-id:4",
            "stats.field",
            "{!key=" + key + " cardinality=true}" + f),
        kpre + "long[@name='cardinality'][.='3']",
        "count(" + kpre + "/*)=1");
  }

  public void doTestMVFieldStatisticsResult(String f) {
    assertU(adoc("id", "1", f, "-10", f, "-100", "active_s", "true"));
    assertU(adoc("id", "2", f, "-20", f, "200", "active_s", "true"));
    assertU(commit());
    assertU(adoc("id", "3", f, "-30", f, "-1", "active_s", "false"));
    assertU(adoc("id", "4", f, "-40", f, "10", "active_s", "false"));
    assertU(adoc("id", "5", "active_s", "false"));
    assertU(adoc("id", "6", "active_s", "false"));
    assertU(adoc("id", "7", "active_s", "true"));

    assertU(commit());

    // with or w/o these excluded filters, results should be the same
    for (SolrParams baseParams :
        new SolrParams[] {
          params("stats.field", f, "stats", "true"),
          params("stats.field", "{!ex=fq1}" + f, "stats", "true", "fq", "{!tag=fq1}id:1"),
          params(
              "stats.field",
              "{!ex=fq1,fq2}" + f,
              "stats",
              "true",
              "fq",
              "{!tag=fq1}-id_i:[0 TO 2]",
              "fq",
              "{!tag=fq2}-id_i:[2 TO 1000]"),
          params(
              "json.facet", // note: no distinctValues support and not comparing min/max values
              "{min:'min("
                  + f
                  + ")',count:'countvals("
                  + f
                  + ")',missing:'missing("
                  + f
                  + ")',max:'max("
                  + f
                  + ")', sum:'sum("
                  + f
                  + ")', "
                  + " countDistinct:'unique("
                  + f
                  + ")', sumOfSquares:'sumsq("
                  + f
                  + ")', mean:'avg("
                  + f
                  + ")', stddev:'stddev("
                  + f
                  + ")' }")
        }) {
      // easy switch to know if/when we are using json.facet which doesn't support some options
      final boolean json = (null != baseParams.get("json.facet"));
      assertQ(
          "test statistics values",
          req(baseParams, "q", "*:*", "stats.calcdistinct", "true"),
          json ? "//*" : "//double[@name='min'][.='-100.0']",
          json ? "//*" : "//double[@name='max'][.='200.0']",
          "//double[@name='sum'][.='9.0']",
          "//long[@name='count'][.='8']",
          "//long[@name='missing'][.='3']",
          "//long[@name='countDistinct'][.='8']",
          json ? "//*" : "count(//arr[@name='distinctValues']/*)=8",
          "//double[@name='sumOfSquares'][.='53101.0']",
          "//double[@name='mean'][.='1.125']",
          "//double[@name='stddev'][.='87.08852228787508']");

      assertQ(
          "test statistics values w/fq",
          req(baseParams, "fq", "-id:1", "q", "*:*", "stats.calcdistinct", "true"),
          json ? "//*" : "//double[@name='min'][.='-40.0']",
          json ? "//*" : "//double[@name='max'][.='200.0']",
          "//double[@name='sum'][.='119.0']",
          "//long[@name='count'][.='6']",
          "//long[@name='missing'][.='3']",
          "//long[@name='countDistinct'][.='6']",
          json ? "//*" : "count(//arr[@name='distinctValues']/*)=6",
          "//double[@name='sumOfSquares'][.='43001.0']",
          "//double[@name='mean'][.='19.833333333333332']",
          "//double[@name='stddev'][.='90.15634568163611']");

      assertQ(
          "test stdDev",
          req(baseParams, "q", "id:5", "rows", "0"),
          "//double[@name='stddev'][.='0.0']");

      if (!json) { // checking stats.facet makes no sense for json faceting
        assertQ(
            "test stats.facet (using boolean facet field)",
            req(baseParams, "q", "*:*", "stats.calcdistinct", "true", "stats.facet", "active_s"),
            // baseline
            "//lst[@name='" + f + "']/double[@name='min'][.='-100.0']",
            "//lst[@name='" + f + "']/double[@name='max'][.='200.0']",
            "//lst[@name='" + f + "']/double[@name='sum'][.='9.0']",
            "//lst[@name='" + f + "']/long[@name='count'][.='8']",
            "//lst[@name='" + f + "']/long[@name='missing'][.='3']",
            "//lst[@name='" + f + "']/long[@name='countDistinct'][.='8']",
            "count(//lst[@name='" + f + "']/arr[@name='distinctValues']/*)=8",
            "//lst[@name='" + f + "']/double[@name='sumOfSquares'][.='53101.0']",
            "//lst[@name='" + f + "']/double[@name='mean'][.='1.125']",
            "//lst[@name='" + f + "']/double[@name='stddev'][.='87.08852228787508']",
            // facet 'true'
            "//lst[@name='true']/double[@name='min'][.='-100.0']",
            "//lst[@name='true']/double[@name='max'][.='200.0']",
            "//lst[@name='true']/double[@name='sum'][.='70.0']",
            "//lst[@name='true']/long[@name='count'][.='4']",
            "//lst[@name='true']/long[@name='missing'][.='1']",
            "//lst[@name='true']//long[@name='countDistinct'][.='4']",
            "count(//lst[@name='true']/arr[@name='distinctValues']/*)=4",
            "//lst[@name='true']/double[@name='sumOfSquares'][.='50500.0']",
            "//lst[@name='true']/double[@name='mean'][.='17.5']",
            "//lst[@name='true']/double[@name='stddev'][.='128.16005617976296']",
            // facet 'false'
            "//lst[@name='false']/double[@name='min'][.='-40.0']",
            "//lst[@name='false']/double[@name='max'][.='10.0']",
            "//lst[@name='false']/double[@name='sum'][.='-61.0']",
            "//lst[@name='false']/long[@name='count'][.='4']",
            "//lst[@name='false']/long[@name='missing'][.='2']",
            "//lst[@name='true']//long[@name='countDistinct'][.='4']",
            "count(//lst[@name='true']/arr[@name='distinctValues']/*)=4",
            "//lst[@name='false']/double[@name='sumOfSquares'][.='2601.0']",
            "//lst[@name='false']/double[@name='mean'][.='-15.25']",
            "//lst[@name='false']/double[@name='stddev'][.='23.59908190304586']");
      }
    }

    // cardinality
    for (SolrParams baseParams :
        new SolrParams[] {
          params("stats.field", "{!cardinality=true}" + f, "stats", "true"),
          params("json.facet", "{cardinality:'hll(" + f + ")'}")
        }) {
      assertQ(
          "test cardinality",
          req(baseParams, "q", "*:*", "rows", "0"),
          "//long[@name='cardinality'][.='8']");
    }
  }

  public void testFieldStatisticsResultsStringField() {
    String f = "active_s";

    assertU(adoc("id", "1", f, "string1"));
    assertU(adoc("id", "2", f, "string2"));
    assertU(adoc("id", "3", f, "string3"));
    assertU(adoc("id", "4"));
    assertU(commit());

    for (SolrParams baseParams :
        new SolrParams[] {
          params(
              StatsParams.STATS_FIELD,
              f,
              StatsParams.STATS,
              "true",
              "f." + f + ".stats.calcdistinct",
              "true"),
          params(
              "json.facet", // note: no distinctValues support
              "{min:'min("
                  + f
                  + ")',count:'countvals("
                  + f
                  + ")',missing:'missing("
                  + f
                  + ")',max:'max("
                  + f
                  + ")', "
                  + " countDistinct:'unique("
                  + f
                  + ")'}")
        }) {
      final boolean json = (null != baseParams.get("json.facet"));
      assertQ(
          "test string statistics values",
          req(baseParams, "q", "*:*", "rows", "0"),
          "//str[@name='min'][.='string1']",
          "//str[@name='max'][.='string3']",
          "//long[@name='count'][.='3']",
          "//long[@name='missing'][.='1']",
          "//long[@name='countDistinct'][.='3']",
          json ? "//*" : "count(//arr[@name='distinctValues']/str)=3"); // SOLR-14011
    }

    // string field cardinality
    for (SolrParams baseParams :
        new SolrParams[] {
          params(StatsParams.STATS_FIELD, "{!cardinality=true}" + f, StatsParams.STATS, "true"),
          params("json.facet", "{cardinality:'hll(" + f + ")'}")
        }) {
      assertQ(
          "test string cardinality",
          req(baseParams, "q", "*:*", "rows", "0"),
          "//long[@name='cardinality'][.='3']");
    }

    String strFunc = "strdist(\"string22\"," + f + ",edit)";
    // stats over a string function
    for (SolrParams baseParams :
        new SolrParams[] {
          params(StatsParams.STATS_FIELD, "{!func}" + strFunc, StatsParams.STATS, "true"),
          params(
              "json.facet", // note: no function support for unique
              "{min:'min("
                  + strFunc
                  + ")',count:'countvals("
                  + strFunc
                  + ")',missing:'missing("
                  + strFunc
                  + ")',"
                  + "sum:'sum("
                  + strFunc
                  + ")', max:'max("
                  + strFunc
                  + ")'}")
        }) {
      final boolean json = (null != baseParams.get("json.facet"));
      assertQ(
          "strdist func stats",
          req(baseParams, "q", "*:*", "rows", "0"),
          "//double[@name='min'][.='0.75']",
          "//double[@name='max'][.='0.875']",
          "//double[@name='sum'][.='2.375']",
          json ? "//*" : "//long[@name='count'][.='3']", // SOLR-14010
          "//long[@name='missing'][.='1']");
    }
  }

  public void testFieldStatisticsResultsDateField() {
    SolrCore core = h.getCore();

    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT);
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

    String date1 = dateFormat.format(new Date(123456789)) + "Z";
    String date2 = dateFormat.format(new Date(987654321)) + "Z";

    assertU(adoc("id", "1", "active_dt", date1));
    assertU(adoc("id", "2", "active_dt", date2));
    assertU(adoc("id", "3"));
    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "active_dt");
    args.put("f.active_dt.stats.calcdistinct", "true");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test date statistics values",
        req,
        "//long[@name='count'][.='2']",
        "//long[@name='missing'][.='1']",
        "//date[@name='min'][.='1970-01-02T10:17:36Z']",
        "//date[@name='max'][.='1970-01-12T10:20:54Z']",
        "//long[@name='countDistinct'][.='2']",
        "count(//arr[@name='distinctValues']/date)=2"
        //  "//date[@name='sum'][.='1970-01-13T20:38:30Z']",  // sometimes 29.999Z
        //  "//date[@name='mean'][.='1970-01-07T10:19:15Z']"  // sometiems 14.999Z
        );

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}active_dt"),
        "//lst[@name='active_dt']/long[@name='cardinality'][.='2']");
  }

  // Check for overflow of sumOfSquares
  public void testFieldStatisticsResultsDateFieldOverflow() {
    SolrCore core = h.getCore();

    assertU(adoc("id", "1", "active_dt", "2015-12-14T09:00:00Z"));
    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "active_dt");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test date statistics values",
        req,
        "//long[@name='count'][.='1']",
        "//date[@name='min'][.='2015-12-14T09:00:00Z']",
        "//date[@name='max'][.='2015-12-14T09:00:00Z']",
        "//double[@name='sum'][.='1.4500836E12']",
        "//date[@name='mean'][.='2015-12-14T09:00:00Z']",
        "//double[@name='sumOfSquares'][.='" + Double.toString(2102742446988960000000000.0) + "']");

    assertU(adoc("id", "2", "active_dt", "2115-12-14T09:00:00Z"));
    assertU(adoc("id", "3", "active_dt", "2215-12-14T09:00:00Z"));
    assertU(commit());

    assertQ(
        "test date statistics values",
        req,
        "//long[@name='count'][.='3']",
        "//date[@name='min'][.='2015-12-14T09:00:00Z']",
        "//date[@name='max'][.='2215-12-14T09:00:00Z']",
        "//double[@name='sum'][.='1.38172716E13']",
        "//date[@name='mean'][.='2115-12-14T09:00:00Z']",
        "//double[@name='sumOfSquares'][.='" + Double.toString(83555549895529430000000000.0) + "']",
        // The following number matches the number returned by the current solr
        // implementation of standard deviation. Should be 3155673600000.
        // That number is not precise, and the implementation should be fixed.
        "//double[@name='stddev'][.='" + Double.toString(3155673599999.999) + "']");
  }

  public void doTestFieldStatisticsMissingResult(String f, SolrParams[] baseParamsSet) {
    assertU(adoc("id", "1", f, "-10"));
    assertU(adoc("id", "2", f, "-20"));
    assertU(commit());
    assertU(adoc("id", "3"));
    assertU(adoc("id", "4", f, "-40"));
    assertU(commit());

    final String key = "key_key";
    final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='" + key + "']/";

    // status should be the same regardless of baseParams
    for (SolrParams baseParams : baseParamsSet) {

      SolrQueryRequest request = req(baseParams, "q", "*:*", "stats.calcdistinct", "true");

      assertQ(
          "test statistics values",
          request,
          "//double[@name='min'][.='-40.0']",
          "//double[@name='max'][.='-10.0']",
          "//double[@name='sum'][.='-70.0']",
          "//long[@name='count'][.='3']",
          "//long[@name='missing'][.='1']",
          "//long[@name='countDistinct'][.='3']",
          "count(//arr[@name='distinctValues']/*)=3",
          "//double[@name='sumOfSquares'][.='2100.0']",
          "//double[@name='mean'][.='-23.333333333333332']",
          "//double[@name='stddev'][.='15.275252316519467']");
    }

    // we should be able to compute exact same stats for a field even
    // when we specify it using the "field()" function, or use other
    // identify equivalent functions
    for (String param :
        new String[] {
          // bare
          "{!key=" + key + " ex=key_ex_tag}" + f,
          "{!key=" + key + " ex=key_ex_tag v=" + f + "}",
          // field func
          "{!lucene key=" + key + " ex=key_ex_tag}_val_:\"field(" + f + ")\"",
          "{!func key=" + key + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + key + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + key + " ex=key_ex_tag v=field(" + f + ")}",
          "{!type=func key=" + key + " ex=key_ex_tag v='field(" + f + ")'}",
          // identity math functions
          "{!type=func key=" + key + " ex=key_ex_tag v='sum(0," + f + ")'}",
          "{!type=func key=" + key + " ex=key_ex_tag v='product(1," + f + ")'}",
        }) {

      assertQ(
          "test statistics over field specified as a function: " + param,
          // NOTE: baseParams aren't used, we're looking at the function
          req(
              "q",
              "*:*",
              "stats",
              "true",
              "stats.calcdistinct",
              "true",
              "fq",
              "{!tag=key_ex_tag}-id:4",
              "stats.field",
              param),
          kpre + "double[@name='min'][.='-40.0']",
          kpre + "double[@name='max'][.='-10.0']",
          kpre + "double[@name='sum'][.='-70.0']",
          kpre + "long[@name='count'][.='3']",
          kpre + "long[@name='missing'][.='1']",
          kpre + "long[@name='countDistinct'][.='3']",
          "count(" + kpre + "arr[@name='distinctValues']/*)=3",
          kpre + "double[@name='sumOfSquares'][.='2100.0']",
          kpre + "double[@name='mean'][.='-23.333333333333332']",
          kpre + "double[@name='stddev'][.='15.275252316519467']");
    }
  }

  public void doTestFacetStatisticsResult(String f, SolrParams[] baseParamsSet) {
    assertU(adoc("id", "1", f, "10", "active_s", "true", "other_s", "foo"));
    assertU(adoc("id", "2", f, "20", "active_s", "true", "other_s", "bar"));
    assertU(commit());
    assertU(adoc("id", "3", f, "30", "active_s", "false", "other_s", "foo"));
    assertU(adoc("id", "4", f, "40", "active_s", "false", "other_s", "foo"));
    assertU(commit());

    final String pre =
        "//lst[@name='stats_fields']/lst[@name='"
            + f
            + "']/lst[@name='facets']/lst[@name='active_s']";

    // status should be the same regardless of baseParams
    for (SolrParams baseParams : baseParamsSet) {

      assertQ(
          "test value for active_s=true",
          req(
              baseParams,
              "q",
              "*:*",
              "stats.calcdistinct",
              "true",
              "stats.facet",
              "active_s",
              "stats.facet",
              "other_s"),
          "*[count(" + pre + ")=1]",
          pre + "/lst[@name='true']/double[@name='min'][.='10.0']",
          pre + "/lst[@name='true']/double[@name='max'][.='20.0']",
          pre + "/lst[@name='true']/double[@name='sum'][.='30.0']",
          pre + "/lst[@name='true']/long[@name='count'][.='2']",
          pre + "/lst[@name='true']/long[@name='missing'][.='0']",
          pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']",
          "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2",
          pre + "/lst[@name='true']/double[@name='sumOfSquares'][.='500.0']",
          pre + "/lst[@name='true']/double[@name='mean'][.='15.0']",
          pre + "/lst[@name='true']/double[@name='stddev'][.='7.0710678118654755']");

      assertQ(
          "test value for active_s=false",
          req(baseParams, "q", "*:*", "stats.calcdistinct", "true", "stats.facet", "active_s"),
          pre + "/lst[@name='false']/double[@name='min'][.='30.0']",
          pre + "/lst[@name='false']/double[@name='max'][.='40.0']",
          pre + "/lst[@name='false']/double[@name='sum'][.='70.0']",
          pre + "/lst[@name='false']/long[@name='count'][.='2']",
          pre + "/lst[@name='false']/long[@name='missing'][.='0']",
          pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']",
          "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2",
          pre + "/lst[@name='false']/double[@name='sumOfSquares'][.='2500.0']",
          pre + "/lst[@name='false']/double[@name='mean'][.='35.0']",
          pre + "/lst[@name='false']/double[@name='stddev'][.='7.0710678118654755']");
    }

    // we should be able to compute exact same stats & stats.facet for a field even
    // when we specify it using the "field()" function, or use other
    // identify equivalent functions
    for (String param :
        new String[] {
          // bare
          "{!key=" + f + " ex=key_ex_tag}" + f,
          "{!key=" + f + " ex=key_ex_tag v=" + f + "}",
          // field func
          "{!lucene key=" + f + " ex=key_ex_tag}_val_:\"field(" + f + ")\"",
          "{!func key=" + f + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + f + " ex=key_ex_tag}field(" + f + ")",
          "{!type=func key=" + f + " ex=key_ex_tag v=field(" + f + ")}",
          "{!type=func key=" + f + " ex=key_ex_tag v='field(" + f + ")'}",
          // identity math functions
          "{!type=func key=" + f + " ex=key_ex_tag v='sum(0," + f + ")'}",
          "{!type=func key=" + f + " ex=key_ex_tag v='product(1," + f + ")'}",
        }) {
      assertQ(
          "test stats & stats.facet over field specified as a function: " + param,
          req(
              "q",
              "*:*",
              "stats",
              "true",
              "stats.calcdistinct",
              "true",
              "fq",
              "{!tag=key_ex_tag}-id:4",
              "stats.field",
              param,
              "stats.facet",
              "active_s",
              "stats.facet",
              "other_s"),
          "*[count(" + pre + ")=1]",
          pre + "/lst[@name='true']/double[@name='min'][.='10.0']",
          pre + "/lst[@name='true']/double[@name='max'][.='20.0']",
          pre + "/lst[@name='true']/double[@name='sum'][.='30.0']",
          pre + "/lst[@name='true']/long[@name='count'][.='2']",
          pre + "/lst[@name='true']/long[@name='missing'][.='0']",
          pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']",
          "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2",
          pre + "/lst[@name='true']/double[@name='sumOfSquares'][.='500.0']",
          pre + "/lst[@name='true']/double[@name='mean'][.='15.0']",
          pre + "/lst[@name='true']/double[@name='stddev'][.='7.0710678118654755']",
          pre + "/lst[@name='false']/double[@name='min'][.='30.0']",
          pre + "/lst[@name='false']/double[@name='max'][.='40.0']",
          pre + "/lst[@name='false']/double[@name='sum'][.='70.0']",
          pre + "/lst[@name='false']/long[@name='count'][.='2']",
          pre + "/lst[@name='false']/long[@name='missing'][.='0']",
          pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']",
          "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2",
          pre + "/lst[@name='false']/double[@name='sumOfSquares'][.='2500.0']",
          pre + "/lst[@name='false']/double[@name='mean'][.='35.0']",
          pre + "/lst[@name='false']/double[@name='stddev'][.='7.0710678118654755']");
    }

    assertQ(
        "stats.facet w/ cardinality",
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "fq",
            "-other_s:bar",
            "stats.facet",
            "active_s",
            "stats.field",
            "{!cardinality=true}" + f),
        pre + "/lst[@name='true' ]/long[@name='cardinality'][.='1']",
        pre + "/lst[@name='false']/long[@name='cardinality'][.='2']");
  }

  public void doTestFacetStatisticsMissingResult(String f, SolrParams[] baseParamsSet) {
    assertU(adoc("id", "1", f, "10", "active_s", "true"));
    assertU(adoc("id", "2", f, "20", "active_s", "true"));
    assertU(commit());
    assertU(adoc("id", "3", "active_s", "false"));
    assertU(adoc("id", "4", f, "40", "active_s", "false"));
    assertU(commit());

    // status should be the same regardless of baseParams
    for (SolrParams baseParams : baseParamsSet) {

      assertQ(
          "test value for active_s=true",
          req(baseParams, "q", "*:*", "stats.calcdistinct", "true", "stats.facet", "active_s"),
          "//lst[@name='true']/double[@name='min'][.='10.0']",
          "//lst[@name='true']/double[@name='max'][.='20.0']",
          "//lst[@name='true']/double[@name='sum'][.='30.0']",
          "//lst[@name='true']/long[@name='count'][.='2']",
          "//lst[@name='true']/long[@name='missing'][.='0']",
          "//lst[@name='true']/long[@name='countDistinct'][.='2']",
          "count(//lst[@name='true']/arr[@name='distinctValues']/*)=2",
          "//lst[@name='true']/double[@name='sumOfSquares'][.='500.0']",
          "//lst[@name='true']/double[@name='mean'][.='15.0']",
          "//lst[@name='true']/double[@name='stddev'][.='7.0710678118654755']");

      assertQ(
          "test value for active_s=false",
          req(baseParams, "q", "*:*", "stats.facet", "active_s", "stats.calcdistinct", "true"),
          "//lst[@name='false']/double[@name='min'][.='40.0']",
          "//lst[@name='false']/double[@name='max'][.='40.0']",
          "//lst[@name='false']/double[@name='sum'][.='40.0']",
          "//lst[@name='false']/long[@name='count'][.='1']",
          "//lst[@name='false']/long[@name='missing'][.='1']",
          "//lst[@name='false']/long[@name='countDistinct'][.='1']",
          "count(//lst[@name='false']/arr[@name='distinctValues']/*)=1",
          "//lst[@name='false']/double[@name='sumOfSquares'][.='1600.0']",
          "//lst[@name='false']/double[@name='mean'][.='40.0']",
          "//lst[@name='false']/double[@name='stddev'][.='0.0']");
    }

    assertQ(
        "stats.facet w/ cardinality",
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.facet",
            "active_s",
            "stats.field",
            "{!cardinality=true}" + f),
        "//lst[@name='active_s']/lst[@name='true' ]/long[@name='cardinality'][.='2']",
        "//lst[@name='active_s']/lst[@name='false']/long[@name='cardinality'][.='1']");
  }

  public void testFieldStatisticsResultsNumericFieldAlwaysMissing() {
    SolrCore core = h.getCore();
    assertU(adoc("id", "1"));
    assertU(adoc("id", "2"));
    assertU(commit());
    assertU(adoc("id", "3"));
    assertU(adoc("id", "4"));
    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "active_i");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test string statistics values",
        req,
        "//lst[@name='active_i']/long[@name='count'][.='0']",
        "//lst[@name='active_i']/long[@name='missing'][.='4']",
        "//lst[@name='active_i']/null[@name='min']",
        "//lst[@name='active_i']/null[@name='max']",
        "//lst[@name='active_i']/double[@name='sum'][.='0.0']",
        "//lst[@name='active_i']/double[@name='sumOfSquares'][.='0.0']",
        "//lst[@name='active_i']/double[@name='stddev'][.='0.0']",
        "//lst[@name='active_i']/double[@name='mean'][.='NaN']",
        // if new stats are supported, this will break - update test to assert values for each
        "count(//lst[@name='active_i']/*)=8");

    // NOTE: empty set percentiles covered in testPercentiles()

    assertQ(
        "test cardinality of missing",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}active_i"),
        "//lst[@name='active_i']/long[@name='cardinality'][.='0']");
  }

  public void testFieldStatisticsResultsStringFieldAlwaysMissing() {
    SolrCore core = h.getCore();
    assertU(adoc("id", "1"));
    assertU(adoc("id", "2"));
    assertU(commit());
    assertU(adoc("id", "3"));
    assertU(adoc("id", "4"));
    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "active_s");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test string statistics values",
        req,
        "//lst[@name='active_s']/long[@name='count'][.='0']",
        "//lst[@name='active_s']/long[@name='missing'][.='4']",
        "//lst[@name='active_s']/null[@name='min']",
        "//lst[@name='active_s']/null[@name='max']",
        // if new stats are supported, this will break - update test to assert values for each
        "count(//lst[@name='active_s']/*)=4");

    assertQ(
        "test string statistics values",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}active_s"),
        "//lst[@name='active_s']/long[@name='cardinality'][.='0']");
  }

  // SOLR-3160
  public void testFieldStatisticsResultsDateFieldAlwaysMissing() {
    SolrCore core = h.getCore();

    assertU(adoc("id", "1"));
    assertU(adoc("id", "2"));
    assertU(commit());
    assertU(adoc("id", "3"));
    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "active_dt");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test string statistics values",
        req,
        "//lst[@name='active_dt']/long[@name='count'][.='0']",
        "//lst[@name='active_dt']/long[@name='missing'][.='3']",
        "//lst[@name='active_dt']/null[@name='min']",
        "//lst[@name='active_dt']/null[@name='max']",
        "//lst[@name='active_dt']/null[@name='mean']",
        "//lst[@name='active_dt']/double[@name='sum'][.='0.0']",
        "//lst[@name='active_dt']/double[@name='sumOfSquares'][.='0.0']",
        "//lst[@name='active_dt']/double[@name='stddev'][.='0.0']",

        // if new stats are supported, this will break - update test to assert values for each
        "count(//lst[@name='active_dt']/*)=8");

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}active_dt"),
        "//lst[@name='active_dt']/long[@name='cardinality'][.='0']");
  }

  public void testStatsFacetMultivaluedErrorHandling() {
    SolrCore core = h.getCore();
    SchemaField foo_ss = core.getLatestSchema().getField("foo_ss");

    assertU(adoc("id", "1", "active_i", "1", "foo_ss", "aa"));
    assertU(commit());
    assertU(adoc("id", "2", "active_i", "1", "foo_ss", "bb"));
    assertU(adoc("id", "3", "active_i", "5", "foo_ss", "aa"));
    assertU(commit());

    assertTrue(
        "schema no longer satisfies test requirements: foo_ss no longer multivalued",
        foo_ss.multiValued());
    assertFalse(
        "schema no longer satisfies test requirements: foo_ss's fieldtype no longer single valued",
        foo_ss.getType().isMultiValued());

    assertQEx(
        "no failure trying to get stats facet on foo_ss",
        req(
            "q", "*:*",
            "stats", "true",
            "stats.field", "active_i",
            "stats.facet", "foo_ss"),
        400);
  }

  // SOLR-3177
  public void testStatsExcludeFilterQuery() {
    SolrCore core = h.getCore();
    assertU(adoc("id", "1"));
    assertU(adoc("id", "2"));
    assertU(adoc("id", "3"));
    assertU(adoc("id", "4"));
    assertU(commit());

    Map<String, String> args = new HashMap<String, String>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "{!ex=id}id_i");
    args.put("fq", "{!tag=id}id_i:[2 TO 3]");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test excluding filter query",
        req,
        "//lst[@name='id_i']/double[@name='min'][.='1.0']",
        "//lst[@name='id_i']/double[@name='max'][.='4.0']");

    args = new HashMap<String, String>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "{!key=id2}id_i");
    args.put("fq", "{!tag=id}id_i:[2 TO 3]");
    req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test rename field",
        req,
        "//lst[@name='id2']/double[@name='min'][.='2.0']",
        "//lst[@name='id2']/double[@name='max'][.='3.0']");
  }

  // SOLR-6024
  public void testFieldStatisticsDocValuesAndMultiValued() throws Exception {
    SolrCore core = h.getCore();

    // precondition for the test
    SchemaField catDocValues = core.getLatestSchema().getField("cat_docValues");
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer multivalued",
        catDocValues.multiValued());
    assertFalse(
        "schema no longer satisfies test requirements: cat_docValues fieldtype no longer single valued",
        catDocValues.getType().isMultiValued());
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer has docValues",
        catDocValues.hasDocValues());

    List<FldType> types = new ArrayList<>();
    types.add(new FldType("id", ONE_ONE, new SVal('A', 'Z', 4, 4)));
    types.add(new FldType("cat_docValues", new IRange(2, 2), new SVal('a', 'z', 1, 30)));
    Doc d1 = createDoc(types);
    d1.getValues("id").set(0, "1");
    d1.getValues("cat_docValues").set(0, "test");
    d1.getValues("cat_docValues").set(1, "testtw");
    updateJ(toJSON(d1), null);
    Doc d2 = createDoc(types);
    d2.getValues("id").set(0, "2");
    d2.getValues("cat_docValues").set(0, "test");
    d2.getValues("cat_docValues").set(1, "testtt");
    updateJ(toJSON(d2), null);

    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, "cat_docValues");
    args.put("indent", "true");
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test min/max on docValues and multiValued",
        req,
        "//lst[@name='cat_docValues']/str[@name='min'][.='test']",
        "//lst[@name='cat_docValues']/str[@name='max'][.='testtw']");

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}cat_docValues"),
        "//lst[@name='cat_docValues']/long[@name='cardinality'][.='3']");
  }

  public void testFieldStatisticsDocValuesAndMultiValuedInteger() throws Exception {
    SolrCore core = h.getCore();
    String fieldName = "cat_intDocValues";
    // precondition for the test
    SchemaField catDocValues = core.getLatestSchema().getField(fieldName);
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer multivalued",
        catDocValues.multiValued());
    assertFalse(
        "schema no longer satisfies test requirements: cat_docValues fieldtype no longer single valued",
        catDocValues.getType().isMultiValued());
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer has docValues",
        catDocValues.hasDocValues());

    List<FldType> types = new ArrayList<>();
    types.add(new FldType("id", ONE_ONE, new SVal('A', 'Z', 4, 4)));
    types.add(new FldType(fieldName, ONE_ONE, new IRange(0, 0)));

    Doc d1 = createDocValuesDocument(types, fieldName, "1", -1, 3, 5);
    updateJ(toJSON(d1), null);

    Doc d2 = createDocValuesDocument(types, fieldName, "2", 3, -2, 6);
    updateJ(toJSON(d2), null);

    Doc d3 = createDocValuesDocument(types, fieldName, "3", 16, -3, 11);
    updateJ(toJSON(d3), null);

    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, fieldName);
    args.put(StatsParams.STATS_CALC_DISTINCT, "true");
    args.put("indent", "true");

    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQ(
        "test min/max on docValues and multiValued",
        req,
        "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']",
        "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']",
        "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']",
        "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']",
        "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']",
        "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']",
        "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']",
        "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']",
        "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']");

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}" + fieldName),
        "//lst[@name='" + fieldName + "']/long[@name='cardinality'][.='9']");
  }

  public void testFieldStatisticsDocValuesAndMultiValuedIntegerFacetStats() throws Exception {
    SolrCore core = h.getCore();
    String fieldName = "cat_intDocValues";
    // precondition for the test
    SchemaField catDocValues = core.getLatestSchema().getField(fieldName);
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer multivalued",
        catDocValues.multiValued());
    assertFalse(
        "schema no longer satisfies test requirements: cat_docValues fieldtype no longer single valued",
        catDocValues.getType().isMultiValued());
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer has docValues",
        catDocValues.hasDocValues());

    List<FldType> types = new ArrayList<>();
    types.add(new FldType("id", ONE_ONE, new SVal('A', 'Z', 4, 4)));
    types.add(new FldType(fieldName, ONE_ONE, new IRange(0, 0)));

    Doc d1 = createDocValuesDocument(types, fieldName, "1", -1, 3, 5);
    updateJ(toJSON(d1), null);

    Doc d2 = createDocValuesDocument(types, fieldName, "2", 3, -2, 6);
    updateJ(toJSON(d2), null);

    Doc d3 = createDocValuesDocument(types, fieldName, "3", 16, -3, 11);
    updateJ(toJSON(d3), null);

    assertU(commit());

    Map<String, String> args = new HashMap<>();
    args.put(CommonParams.Q, "*:*");
    args.put(StatsParams.STATS, "true");
    args.put(StatsParams.STATS_FIELD, fieldName);
    args.put(StatsParams.STATS_FACET, fieldName);
    args.put(StatsParams.STATS_CALC_DISTINCT, "true");
    args.put("indent", "true");

    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));

    assertQEx("can not use FieldCache on multivalued field: cat_intDocValues", req, 400);
  }

  public void testMiscQueryStats() {
    final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";

    assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "foo_t", "how now brown cow"));
    assertU(adoc("id", "2", "a_f", "4.5", "b_f", "8.6", "foo_t", "cow cow cow cow"));
    assertU(adoc("id", "3", "a_f", "5.6", "b_f", "7.5", "foo_t", "red fox")); // no cow
    assertU(adoc("id", "4", "a_f", "6.7", "b_f", "6.3", "foo_t", "red cow"));
    assertU(commit());

    assertQ(
        "functions over multiple fields",
        req("q", "foo_t:cow", "stats", "true", "stats.field", "{!func key=k}product(a_f,b_f)"),
        kpre + "double[@name='min'][.='22.309999465942383']",
        kpre + "double[@name='max'][.='42.209999084472656']",
        kpre + "double[@name='sum'][.='103.21999931335449']",
        kpre + "long[@name='count'][.='3']",
        kpre + "long[@name='missing'][.='0']",
        kpre + "double[@name='sumOfSquares'][.='3777.110157933046']",
        kpre + "double[@name='mean'][.='34.40666643778483']",
        kpre + "double[@name='stddev'][.='10.622007151430441']");

    // force constant score for matches, so we aren't dependent on similarity
    final float constScore = 4.2F;
    final double expectedScore = (double) constScore;
    assertQ(
        "functions over a query",
        req("q", "*:*", "stats", "true", "stats.field", "{!lucene key=k}foo_t:cow^=" + constScore),
        kpre + "double[@name='min'][.='" + expectedScore + "']",
        kpre + "double[@name='max'][.='" + expectedScore + "']",
        kpre + "double[@name='sum'][.='" + (3D * expectedScore) + "']",
        kpre + "long[@name='count'][.='3']",
        kpre + "long[@name='missing'][.='1']",
        kpre + "double[@name='sumOfSquares'][.='" + (3D * Math.pow(expectedScore, 2D)) + "']",
        kpre + "double[@name='mean'][.='" + expectedScore + "']",
        kpre + "double[@name='stddev'][.='0.0']");
  }

  /**
   * Whitebox test of {@link StatsField} parsing to ensure expected equivalence operations hold up
   */
  public void testStatsFieldWhitebox() {
    StatsComponent component = new StatsComponent();
    List<SearchComponent> components = new ArrayList<>(1);
    components.add(component);
    SolrParams common = params("stats", "true", "q", "*:*", "nested", "foo_t:cow");

    // all of these should produce the same SchemaField based StatsField
    for (String param :
        new String[] {"foo_i", "{!func}field(\"foo_i\")", "{!lucene}_val_:\"field(foo_i)\""}) {
      try (SolrQueryRequest req = req(common)) {
        ResponseBuilder rb = new ResponseBuilder(req, new SolrQueryResponse(), components);

        StatsField sf = new StatsField(rb, param);

        assertNull("value source of: " + param, sf.getValueSource());
        assertNotNull("schema field of: " + param, sf.getSchemaField());

        assertEquals("field name of: " + param, "foo_i", sf.getSchemaField().getName());
      }
    }

    // all of these should produce the same QueryValueSource based StatsField
    for (String param :
        new String[] {
          "{!lucene}foo_t:cow", "{!func}query($nested)", "{!field f=foo_t}cow",
        }) {
      try (SolrQueryRequest req = req(common)) {
        ResponseBuilder rb = new ResponseBuilder(req, new SolrQueryResponse(), components);

        StatsField sf = new StatsField(rb, param);

        assertNull("schema field of: " + param, sf.getSchemaField());
        assertNotNull("value source of: " + param, sf.getValueSource());
        assertTrue(
            sf.getValueSource().getClass() + " is vs type of: " + param,
            sf.getValueSource() instanceof QueryValueSource);
        QueryValueSource qvs = (QueryValueSource) sf.getValueSource();
        assertEquals("query of :" + param, new TermQuery(new Term("foo_t", "cow")), qvs.getQuery());
      }
    }
  }

  public void testFieldStatisticsDocValuesAndMultiValuedDouble() throws Exception {
    SolrCore core = h.getCore();
    String fieldName = "cat_floatDocValues";
    // precondition for the test
    SchemaField catDocValues = core.getLatestSchema().getField(fieldName);
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer multivalued",
        catDocValues.multiValued());
    assertFalse(
        "schema no longer satisfies test requirements: cat_docValues fieldtype no longer single valued",
        catDocValues.getType().isMultiValued());
    assertTrue(
        "schema no longer satisfies test requirements: cat_docValues no longer has docValues",
        catDocValues.hasDocValues());

    List<FldType> types = new ArrayList<>();
    types.add(new FldType("id", ONE_ONE, new SVal('A', 'Z', 4, 4)));
    types.add(new FldType(fieldName, ONE_ONE, new FVal(0, 0)));

    Doc d1 = createDocValuesDocument(types, fieldName, "1", -1, 3, 5);
    updateJ(toJSON(d1), null);

    Doc d2 = createDocValuesDocument(types, fieldName, "2", 3, -2, 6);
    updateJ(toJSON(d2), null);

    Doc d3 = createDocValuesDocument(types, fieldName, "3", 16, -3, 11);
    updateJ(toJSON(d3), null);

    assertU(commit());

    final SolrParams baseParams =
        params(CommonParams.Q, "*:*", "indent", "true", StatsParams.STATS, "true");

    SolrQueryRequest req1 =
        req(
            baseParams,
            StatsParams.STATS_CALC_DISTINCT,
            "true",
            StatsParams.STATS_FIELD,
            fieldName);
    SolrQueryRequest req2 =
        req(
            baseParams,
            StatsParams.STATS_FIELD,
            "{!min=true, max=true, count=true, sum=true, mean=true, stddev=true, sumOfSquares=true, missing=true, calcdistinct=true}"
                + fieldName);
    SolrQueryRequest req3 =
        req(
            baseParams,
            StatsParams.STATS_FIELD,
            "{!min=true, max=true, count=true, sum=true, mean=true, stddev=true, sumOfSquares=true, missing=true, countDistinct=true, distinctValues=true}"
                + fieldName);

    for (SolrQueryRequest req : new SolrQueryRequest[] {req1, req2, req3}) {
      assertQ(
          "test status on docValues and multiValued: " + req.toString(),
          req,
          "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']",
          "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']",
          "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']",
          "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']",
          "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']",
          "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']",
          "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']",
          "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']",
          "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']",
          // always comes along with countDistinct
          "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/float)=9",
          // if new default stats are added, this will break - update test to assert values for each
          "count(//lst[@name='" + fieldName + "']/*)=10");
    }

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}" + fieldName),
        "//lst[@name='" + fieldName + "']/long[@name='cardinality'][.='9']");
  }

  public void testEnumFieldTypeStatus() {
    clearIndex();

    String fieldName = "severity";
    assertU(adoc("id", "0", fieldName, "Not Available"));
    assertU(adoc("id", "1", fieldName, "Not Available"));
    assertU(adoc("id", "2", fieldName, "Not Available"));
    assertU(adoc("id", "3", fieldName, "Not Available"));
    assertU(adoc("id", "4", fieldName, "Not Available"));
    assertU(adoc("id", "5", fieldName, "Low"));
    assertU(adoc("id", "6", fieldName, "Low"));
    assertU(adoc("id", "7", fieldName, "Low"));
    assertU(adoc("id", "8", fieldName, "Low"));
    assertU(adoc("id", "9", fieldName, "Medium"));
    assertU(adoc("id", "10", fieldName, "Medium"));
    assertU(adoc("id", "11", fieldName, "Medium"));
    assertU(adoc("id", "12", fieldName, "High"));
    assertU(adoc("id", "13", fieldName, "High"));
    assertU(adoc("id", "14", fieldName, "Critical"));

    for (int i = 20; i <= 30; i++) {
      assertU(adoc("id", "" + i));
    }

    assertU(commit());

    assertQ(
        "enum",
        req("q", "*:*", "stats", "true", "stats.field", fieldName),
        "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']",
        "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']",
        "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']",
        "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");

    assertQ(
        "cardinality",
        req("q", "*:*", "stats", "true", "stats.field", "{!cardinality=true}" + fieldName),
        "//lst[@name='" + fieldName + "']/long[@name='cardinality'][.='5']");

    assertQ(
        "enum calcdistinct",
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.field",
            fieldName,
            StatsParams.STATS_CALC_DISTINCT,
            "true"),
        "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']",
        "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']",
        "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']",
        "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='5']",
        "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/*)=5",
        "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");

    final String pre =
        "//lst[@name='stats_fields']/lst[@name='"
            + fieldName
            + "']/lst[@name='facets']/lst[@name='severity']";

    assertQ(
        "enum + stats.facet",
        req("q", "*:*", "stats", "true", "stats.field", fieldName, "stats.facet", fieldName),
        pre + "/lst[@name='High']/str[@name='min'][.='High']",
        pre + "/lst[@name='High']/str[@name='max'][.='High']",
        pre + "/lst[@name='High']/long[@name='count'][.='2']",
        pre + "/lst[@name='High']/long[@name='missing'][.='0']",
        pre + "/lst[@name='Low']/str[@name='min'][.='Low']",
        pre + "/lst[@name='Low']/str[@name='max'][.='Low']",
        pre + "/lst[@name='Low']/long[@name='count'][.='4']",
        pre + "/lst[@name='Low']/long[@name='missing'][.='0']",
        pre + "/lst[@name='Medium']/str[@name='min'][.='Medium']",
        pre + "/lst[@name='Medium']/str[@name='max'][.='Medium']",
        pre + "/lst[@name='Medium']/long[@name='count'][.='3']",
        pre + "/lst[@name='Medium']/long[@name='missing'][.='0']",
        pre + "/lst[@name='Not Available']/str[@name='min'][.='Not Available']",
        pre + "/lst[@name='Not Available']/str[@name='max'][.='Not Available']",
        pre + "/lst[@name='Not Available']/long[@name='count'][.='5']",
        pre + "/lst[@name='Not Available']/long[@name='missing'][.='0']",
        pre + "/lst[@name='Critical']/str[@name='min'][.='Critical']",
        pre + "/lst[@name='Critical']/str[@name='max'][.='Critical']",
        pre + "/lst[@name='Critical']/long[@name='count'][.='1']",
        pre + "/lst[@name='Critical']/long[@name='missing'][.='0']");
  }

  private Doc createDocValuesDocument(
      List<FldType> types,
      String fieldName,
      String id,
      @SuppressWarnings({"rawtypes"}) Comparable... values) {
    Doc doc = createDoc(types);
    doc.getValues("id").set(0, id);
    initMultyValued(doc.getValues(fieldName), values);
    return doc;
  }

  @SuppressWarnings({"rawtypes"})
  private List<Comparable> initMultyValued(
      List<Comparable> cat_docValues, Comparable... comparables) {
    Collections.addAll(cat_docValues, comparables);
    return cat_docValues;
  }

  /** Convenience struct used in {@link #testIndividualStatLocalParams} */
  private static final class ExpectedStat {
    public static final String KPRE = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
    public final Stat stat;
    public final String input;
    public final List<String> perShardXpaths;
    public final List<String> finalXpaths;

    public static final Map<Stat, ExpectedStat> ALL = new LinkedHashMap<Stat, ExpectedStat>();

    private ExpectedStat(
        Stat stat, String input, List<String> perShardXpaths, List<String> finalXpaths) {
      this.stat = stat;
      this.input = input;
      this.perShardXpaths = perShardXpaths;
      this.finalXpaths = finalXpaths;
    }

    public static void createSimple(Stat stat, String input, String type, String result) {
      EnumSet<Stat> deps = stat.getDistribDeps();
      List<String> perShardXpaths = new ArrayList<String>(deps.size());
      String xpath = KPRE + type + "[@name='" + stat + "'][.='" + result + "']";
      for (Stat dep : deps) {
        if (dep.equals(stat)) { // self dependency
          perShardXpaths.add(xpath);
          ;
        } else {
          ExpectedStat expectedDep = ALL.get(dep);
          assertNotNull("can't find dep in ExpectedStat.ALL", expectedDep);
          perShardXpaths.addAll(expectedDep.perShardXpaths);
        }
      }
      ALL.put(
          stat, new ExpectedStat(stat, input, perShardXpaths, Collections.singletonList(xpath)));
    }

    public static void create(
        Stat stat, String input, List<String> perShardXpaths, List<String> finalXpaths) {
      ALL.put(stat, new ExpectedStat(stat, input, perShardXpaths, finalXpaths));
    }
  }

  public void testIndividualStatLocalParams() {
    final String kpre = ExpectedStat.KPRE;

    assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
    assertU(commit());

    SolrCore core = h.getCore();
    SchemaField field = core.getLatestSchema().getField("a_i");
    HllOptions hllOpts = HllOptions.parseHllOptions(params("cardinality", "true"), field);

    HLL hll = hllOpts.newHLL();
    HashFunction hasher = hllOpts.getHasher();

    AVLTreeDigest tdigest = new AVLTreeDigest(100);

    // some quick sanity check assertions...
    // trivial check that we only get the exact 2 we ask for
    assertQ(
        "ask for and get only 2 stats",
        req("q", "*:*", "stats", "true", "stats.field", "{!key=k mean=true min=true}a_i"),
        kpre + "double[@name='mean'][.='9.0']",
        kpre + "double[@name='min'][.='9.0']",
        "count(" + kpre + "*)=2");

    // for stats that are true/false, check false does its job
    assertQ(
        "min=true & max=false: only min should come back",
        req("q", "*:*", "stats", "true", "stats.field", "{!key=k max=false min=true}a_i"),
        kpre + "double[@name='min'][.='9.0']",
        "count(" + kpre + "*)=1");
    assertQ(
        "min=false: localparam stat means ignore default set, "
            + "but since only local param is false no stats should be returned",
        req("q", "*:*", "stats", "true", "stats.field", "{!key=k min=false}a_i"),
        // section of stats for this field should exist ...
        XPRE + "lst[@name='stats_fields']/lst[@name='k']",
        // ...but be empty
        "count(" + kpre + "*)=0");

    double sum = 0;
    double sumOfSquares = 0;
    final int count = 20;
    for (int i = 0; i < count; i++) {
      int a_i = i % 10;
      assertU(
          adoc(
              "id",
              String.valueOf(i),
              "a_f",
              "2.3",
              "b_f",
              "9.7",
              "a_i",
              String.valueOf(a_i),
              "foo_t",
              "how now brown cow"));
      tdigest.add(a_i);
      hll.addRaw(hasher.hashInt(a_i).asLong());
      sum += a_i;
      sumOfSquares += (a_i) * (a_i);
    }
    double stddev = Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)));

    assertU(commit());

    ByteBuffer tdigestBuf = ByteBuffer.allocate(tdigest.smallByteSize());
    tdigest.asSmallBytes(tdigestBuf);
    byte[] hllBytes = hll.toBytes();

    EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);

    ExpectedStat.createSimple(Stat.min, "true", "double", "0.0");
    ExpectedStat.createSimple(Stat.max, "true", "double", "9.0");
    ExpectedStat.createSimple(Stat.missing, "true", "long", "0");
    ExpectedStat.createSimple(Stat.sum, "true", "double", String.valueOf(sum));
    ExpectedStat.createSimple(Stat.count, "true", "long", String.valueOf(count));
    ExpectedStat.createSimple(Stat.mean, "true", "double", String.valueOf(sum / count));
    ExpectedStat.createSimple(Stat.sumOfSquares, "true", "double", String.valueOf(sumOfSquares));
    ExpectedStat.createSimple(Stat.stddev, "true", "double", String.valueOf(stddev));
    final String distinctValsXpath = "count(" + kpre + "arr[@name='distinctValues']/*)=10";
    ExpectedStat.create(
        Stat.distinctValues,
        "true",
        Collections.singletonList(distinctValsXpath),
        Collections.singletonList(distinctValsXpath));
    ExpectedStat.createSimple(Stat.countDistinct, "true", "long", "10");
    final String percentileShardXpath =
        kpre
            + "str[@name='percentiles'][.='"
            + new String(
                Base64.getEncoder().encode(tdigestBuf.array()), StandardCharsets.ISO_8859_1)
            + "']";
    final String p90 = "" + tdigest.quantile(0.90D);
    final String p99 = "" + tdigest.quantile(0.99D);
    ExpectedStat.create(
        Stat.percentiles,
        "'90, 99'",
        Collections.singletonList(percentileShardXpath),
        Arrays.asList(
            "count(" + kpre + "lst[@name='percentiles']/*)=2",
            kpre + "lst[@name='percentiles']/double[@name='90.0'][.=" + p90 + "]",
            kpre + "lst[@name='percentiles']/double[@name='99.0'][.=" + p99 + "]"));
    final String cardinalityShardXpath =
        kpre
            + "str[@name='cardinality'][.='"
            + new String(Base64.getEncoder().encode(hllBytes), StandardCharsets.ISO_8859_1)
            + "']";
    final String cardinalityXpath = kpre + "long[@name='cardinality'][.='10']";
    ExpectedStat.create(
        Stat.cardinality,
        "true",
        Collections.singletonList(cardinalityShardXpath),
        Collections.singletonList(cardinalityXpath));

    // canary in the coal mine
    assertEquals(
        "num of ExpectedStat doesn't match all known stats; "
            + "enum was updated w/o updating test?",
        ExpectedStat.ALL.size(),
        allStats.size());

    // whitebox test: explicitly ask for isShard=true with each individual stat
    for (ExpectedStat expect : ExpectedStat.ALL.values()) {
      Stat stat = expect.stat;

      StringBuilder exclude = new StringBuilder();
      List<String> testXpaths = new ArrayList<String>(5 + expect.perShardXpaths.size());
      testXpaths.addAll(expect.perShardXpaths);

      int numKeysExpected = 0;
      EnumSet<Stat> distribDeps = stat.getDistribDeps();
      for (Stat perShardDep : distribDeps) {
        numKeysExpected++;

        // even if we go out of our way to exclude the dependent stats,
        // the shard should return them since they are a dependency for the requested stat
        if (!stat.equals(perShardDep)) {
          // NOTE: this only works because all the cases where there are distribDeps
          // beyond a self dependency are simple true/false options
          exclude.append(perShardDep + "=false ");
        }
      }
      // we don't want to find anything we aren't expecting
      testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);

      assertQ(
          "ask for only " + stat + ", with isShard=true, and expect only deps: " + distribDeps,
          req(
              "q",
              "*:*",
              "isShard",
              "true",
              "stats",
              "true",
              "stats.field",
              "{!key=k " + exclude + stat + "=" + expect.input + "}a_i"),
          testXpaths.toArray(new String[0]));
    }

    // test all the possible combinations (of all possible sizes) of stats params
    for (int numParams = 1; numParams <= allStats.size(); numParams++) {
      for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
        // EnumSets use natural ordering, we want to randomize the order of the params
        List<Stat> combo = new ArrayList<Stat>(set);
        Collections.shuffle(combo, random());

        StringBuilder paras = new StringBuilder("{!key=k ");
        List<String> testXpaths = new ArrayList<String>(numParams + 5);

        int numKeysExpected = 0;
        for (Stat stat : combo) {
          ExpectedStat expect = ExpectedStat.ALL.get(stat);

          paras.append(stat + "=" + expect.input + " ");

          numKeysExpected++;
          testXpaths.addAll(expect.finalXpaths);
        }

        paras.append("}a_i");

        // we don't want to find anything we aren't expecting
        testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);

        assertQ(
            "ask for and get only: " + combo,
            req("q", "*:*", "stats", "true", "stats.field", paras.toString()),
            testXpaths.toArray(new String[0]));
      }
    }
  }

  // Test for Solr-6349
  public void testCalcDistinctStats() {
    final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
    final String min = "count(" + kpre + "/double[@name='min'])";
    final String countDistinct = "count(" + kpre + "/long[@name='countDistinct'])";
    final String distinctValues = "count(" + kpre + "/arr[@name='distinctValues'])";

    final int count = 20;
    for (int i = 0; i < count; i++) {
      assertU(
          adoc(
              "id",
              String.valueOf(i),
              "a_f",
              "2.3",
              "b_f",
              "9.7",
              "a_i",
              String.valueOf(i % 10),
              "foo_t",
              "how now brown cow"));
    }

    assertU(commit());

    String[] baseParams = new String[] {"q", "*:*", "stats", "true", "indent", "true"};

    for (SolrParams p :
        new SolrParams[] {
          params("stats.field", "{!key=k}a_i"),
          params(StatsParams.STATS_CALC_DISTINCT, "false", "stats.field", "{!key=k}a_i"),
          params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false", "stats.field", "{!key=k}a_i"),
          params(
              StatsParams.STATS_CALC_DISTINCT,
              "true",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k}a_i"),
          params("stats.field", "{!key=k min='true'}a_i"),
          params(
              StatsParams.STATS_CALC_DISTINCT,
              "true",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "true",
              "stats.field",
              "{!key=k min='true' calcdistinct='false'}a_i"),
        }) {

      assertQ(
          "min is either default or explicitly requested; "
              + "countDistinct & distinctValues either default or explicitly prevented",
          req(p, baseParams),
          min + "=1",
          countDistinct + "=0",
          distinctValues + "=0");
    }

    for (SolrParams p :
        new SolrParams[] {
          params(
              "stats.calcdistinct", "true",
              "stats.field", "{!key=k}a_i"),
          params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true", "stats.field", "{!key=k}a_i"),
          params(
              "stats.calcdistinct",
              "false",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "true",
              "stats.field",
              "{!key=k}a_i"),
          params(
              "stats.calcdistinct", "false ",
              "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
          params(
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k min=true calcdistinct=true}a_i"),
          params(
              "stats.calcdistinct",
              "false ",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k min=true calcdistinct=true}a_i"),
        }) {

      assertQ(
          "min is either default or explicitly requested; "
              + "countDistinct & distinctValues explicitly requested",
          req(p, baseParams),
          min + "=1",
          countDistinct + "=1",
          distinctValues + "=1");
    }

    for (SolrParams p :
        new SolrParams[] {
          params("stats.field", "{!key=k calcdistinct=true}a_i"),
          params(
              "stats.calcdistinct", "true",
              "stats.field", "{!key=k min='false'}a_i"),
          params(
              "stats.calcdistinct", "true",
              "stats.field", "{!key=k max='true' min='false'}a_i"),
          params(
              "stats.calcdistinct", "false",
              "stats.field", "{!key=k calcdistinct=true}a_i"),
          params(
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k calcdistinct=true}a_i"),
          params(
              "stats.calcdistinct",
              "false",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k calcdistinct=true}a_i"),
          params(
              "stats.calcdistinct",
              "false",
              "f.a_i." + StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k min='false' calcdistinct=true}a_i"),
        }) {

      assertQ(
          "min is explicitly excluded; " + "countDistinct & distinctValues explicitly requested",
          req(p, baseParams),
          min + "=0",
          countDistinct + "=1",
          distinctValues + "=1");
    }

    for (SolrParams p :
        new SolrParams[] {
          params(StatsParams.STATS_CALC_DISTINCT, "true", "stats.field", "{!key=k min=true}a_i"),
          params(
              "f.a_i.stats.calcdistinct", "true",
              "stats.field", "{!key=k min=true}a_i"),
          params(
              StatsParams.STATS_CALC_DISTINCT,
              "false",
              "f.a_i.stats.calcdistinct",
              "true",
              "stats.field",
              "{!key=k min=true}a_i"),
          params(
              "f.a_i.stats.calcdistinct", "false",
              "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
          params(
              StatsParams.STATS_CALC_DISTINCT,
              "false",
              "stats.field",
              "{!key=k min=true calcdistinct=true}a_i"),
          params(
              StatsParams.STATS_CALC_DISTINCT,
              "false",
              "f.a_i.stats.calcdistinct",
              "false",
              "stats.field",
              "{!key=k min=true calcdistinct=true}a_i"),
        }) {

      assertQ(
          "min is explicitly requested; " + "countDistinct & distinctValues explicitly requested",
          req(p, baseParams),
          min + "=1",
          countDistinct + "=1",
          distinctValues + "=1");
    }
  }

  /** Helper used in {@link #testCardinality} */
  public static String cardinalityXpath(String key, int cardinality) {
    return XPRE
        + "lst[@name='stats_fields']/lst[@name='"
        + key
        + "']/long[@name='cardinality'][.='"
        + cardinality
        + "']";
  }

  /**
   * @see #testHllOptions
   */
  public void testCardinality() {
    SolrCore core = h.getCore();
    // insure we have the same hasher a_l would use
    HashFunction hasher =
        HllOptions.parseHllOptions(
                params("cardinality", "true"), core.getLatestSchema().getField("a_l"))
            .getHasher();

    String[] baseParams = new String[] {"q", "*:*", "stats", "true", "indent", "true", "rows", "0"};
    assertQ(
        "empty cardinalities",
        req(
            params(
                "stats.field", "{!key=a cardinality=true}a_l",
                "stats.field", "{!key=pa cardinality=true}prehashed_a_l",
                "stats.field", "{!key=b cardinality=true}b_l",
                "stats.field", "{!key=c cardinality=true}c_l"),
            baseParams),
        cardinalityXpath("a", 0),
        cardinalityXpath("pa", 0),
        cardinalityXpath("b", 0),
        cardinalityXpath("c", 0));

    int id = 0;
    // add trivial docs to test basic cardinality
    for (int i = 0; i < 100; i++) {
      // add the same values multiple times (diff docs)
      for (int j = 0; j < 5; j++) {
        ++id;
        assertU(
            adoc(
                "id",
                "" + id,
                "a_l",
                "" + i,
                "prehashed_a_l",
                "" + hasher.hashLong((long) i).asLong(),
                "b_l",
                "" + (i % 7),
                "c_l",
                "" + id));
      }
    }
    assertU(commit());

    assertQ(
        "various cardinalities",
        req(
            params(
                "stats.field", "{!key=a cardinality=true}a_l",
                "stats.field", "{!key=pa hllPreHashed=true cardinality=true}prehashed_a_l",
                "stats.field", "{!key=b cardinality=true}b_l",
                "stats.field", "{!key=c cardinality=true}c_l"),
            baseParams),
        cardinalityXpath("a", 100),
        cardinalityXpath("pa", 100),
        cardinalityXpath("b", 7),
        cardinalityXpath("c", 500));

    // various ways of explicitly saying "don't bother to compute cardinality"
    for (SolrParams p :
        new SolrParams[] {
          params("stats.field", "{!key=a min=true cardinality=false}a_l"),
          params("stats.field", "{!key=a min=true cardinality=$doit}a_l", "doit", "false"),
          params("stats.field", "{!key=a min=true cardinality=$doit}a_l"), // missing doit param
          // other tuning options shouldn't change things
          params("stats.field", "{!key=a min=true hllPreHashed=true cardinality=false}a_l"),
          params(
              "stats.field",
              "{!key=a min=true hllRegwidth=4 cardinality=$doit}a_l",
              "doit",
              "false"),
          params(
              "stats.field",
              "{!key=a min=true hllLog2m=18 cardinality=$doit}a_l"), // missing doit param
        }) {
      assertQ(
          "min w/cardinality explicitly disabled",
          req(p, baseParams),
          "count(//lst[@name='stats_fields']/lst[@name='a']/double[@name='min'])=1",
          "count(//lst[@name='stats_fields']/lst[@name='a']/long[@name='cardinality'])=0");
    }
  }

  /**
   * whitebox test that HLL Option parsing does the right thing
   *
   * @see #testCardinality
   * @see #testHllOptionsErrors
   */
  public void testHllOptions() {
    SolrCore core = h.getCore();

    SchemaField field_l = core.getLatestSchema().getField("field_l");
    SchemaField field_d = core.getLatestSchema().getField("field_d");
    SchemaField field_dt = core.getLatestSchema().getField("field_dt");
    SchemaField field_s = core.getLatestSchema().getField("field_s");
    SchemaField field_i = core.getLatestSchema().getField("field_i");
    SchemaField field_f = core.getLatestSchema().getField("field_f");
    SchemaField field_severity = core.getLatestSchema().getField("severity");

    // simple cases that shouldn't use HLL
    assertNull(HllOptions.parseHllOptions(params(), field_l));
    assertNull(HllOptions.parseHllOptions(params("cardinality", "false"), field_l));

    // check, future-proof against the HLL library changing stuff on us
    assertEquals(
        "HLL Changed definition min for log2m, "
            + "need to note in upgrade instructions and maybe adjust accuracy heuristic",
        4,
        HLL.MINIMUM_LOG2M_PARAM);
    // NOTE: https://github.com/aggregateknowledge/java-hll/issues/14
    assertEquals(
        "HLL Changed definition max for log2m, "
            + "need to note in upgrade instructions and maybe adjust accuracy heuristic",
        30,
        HLL.MAXIMUM_LOG2M_PARAM);
    assertEquals(
        "HLL Changed definition min for regwidth, "
            + "need to note in upgrade instructions and probably adjust heuristic",
        1,
        HLL.MINIMUM_REGWIDTH_PARAM);
    assertEquals(
        "HLL Changed definition max for regwidth, "
            + "need to note in upgrade instructions and probably adjust heuristic",
        8,
        HLL.MAXIMUM_REGWIDTH_PARAM);

    // all of these should produce equivalent HLLOptions (Long, Double, or String using defaults)
    SolrParams[] longDefaultParams =
        new SolrParams[] {
          // basic usage
          params("cardinality", "true"),
          params("cardinality", "0.33"),

          // expert level options
          params("cardinality", "true", "hllLog2m", "13"),
          params("cardinality", "true", "hllRegwidth", "6"),
          params("cardinality", "true", "hllPreHash", "false"),
          params(
              "cardinality", "true", "hllLog2m", "13", "hllRegwidth", "6", "hllPreHash", "false"),

          // explicit hllLog2M should override numeric arg
          params("cardinality", "1.0", "hllLog2m", "13", "hllRegwidth", "6"),
          params("cardinality", "0.0", "hllLog2m", "13", "hllRegwidth", "6", "hllPreHash", "false")
        };
    for (SchemaField field : new SchemaField[] {field_l, field_d, field_dt, field_s}) {
      final String f = field.getName();
      for (SolrParams p : longDefaultParams) {
        HllOptions opts = HllOptions.parseHllOptions(p, field);
        assertEquals(f + " long defaults: " + p, 13, opts.getLog2m());
        assertEquals(f + " long defaults: " + p, 6, opts.getRegwidth());
        assertNotNull(f + " long defaults: " + p, opts.getHasher());
      }

      // non defaults: lower/upper accuracy bounds should give min/max log2m & adjusted regwidth
      HllOptions optsMin = HllOptions.parseHllOptions(params("cardinality", "0"), field);
      assertEquals(f + " min log2m", HLL.MINIMUM_LOG2M_PARAM, optsMin.getLog2m());
      assertEquals(f + " min regwidth", 5, optsMin.getRegwidth()); // lowest heuristic for 64bit

      HllOptions optsMax = HllOptions.parseHllOptions(params("cardinality", "1"), field);
      assertEquals(f + " max log2m", HLL.MAXIMUM_LOG2M_PARAM, optsMax.getLog2m());
      assertEquals(f + " max regwidth", HLL.MAXIMUM_REGWIDTH_PARAM, optsMax.getRegwidth());
    }

    // all of these should produce equivalent HLLOptions (Int, Float, or ValueSource using defaults)
    SolrParams[] intDefaultParams =
        new SolrParams[] {
          // basic usage
          params("cardinality", "true"),
          params("cardinality", "0.33"),

          // expert level options
          params("cardinality", "true", "hllLog2m", "13"),
          params("cardinality", "true", "hllRegwidth", "5"),
          params("cardinality", "true", "hllPreHash", "false"),
          params(
              "cardinality", "true", "hllLog2m", "13", "hllRegwidth", "5", "hllPreHash", "false"),

          // explicit hllLog2M & hllRegwidth should override heuristic float arg
          params("cardinality", "1.0", "hllLog2m", "13", "hllRegwidth", "5"),
          params("cardinality", "0.0", "hllLog2m", "13", "hllRegwidth", "5", "hllPreHash", "false")
        };
    for (SchemaField field : new SchemaField[] {field_i, field_f, field_severity, null}) {
      final String f = null == field ? "(func)" : field.getName();
      for (SolrParams p : intDefaultParams) {
        HllOptions opts = HllOptions.parseHllOptions(p, field);
        assertEquals(f + " int defaults: " + p, 13, opts.getLog2m());
        assertEquals(f + " int defaults: " + p, 5, opts.getRegwidth());
        assertNotNull(f + " int defaults: " + p, opts.getHasher());
      }

      // non defaults: lower/upper accuracy bounds should give min/max log2m & adjusted regwidth
      HllOptions optsMin = HllOptions.parseHllOptions(params("cardinality", "0"), field);
      assertEquals(f + " min log2m", HLL.MINIMUM_LOG2M_PARAM, optsMin.getLog2m());
      assertEquals(f + " min regwidth", 4, optsMin.getRegwidth()); // lowest heuristic for 32bit

      HllOptions optsMax = HllOptions.parseHllOptions(params("cardinality", "1"), field);
      assertEquals(f + " max log2m", HLL.MAXIMUM_LOG2M_PARAM, optsMax.getLog2m());
      assertEquals(f + " max regwidth", HLL.MAXIMUM_REGWIDTH_PARAM, optsMax.getRegwidth());
    }

    // basic pre-hashed arg check specifically for long fields
    assertNotNull(HllOptions.parseHllOptions(params("cardinality", "true"), field_l).getHasher());
    assertNotNull(
        HllOptions.parseHllOptions(params("cardinality", "true", "hllPreHashed", "false"), field_l)
            .getHasher());
    assertNull(
        HllOptions.parseHllOptions(params("cardinality", "true", "hllPreHashed", "true"), field_l)
            .getHasher());
  }

  /**
   * Test user input errors (split into its own test to isolate ignored exceptions)
   *
   * @see #testCardinality
   * @see #testHllOptions
   */
  public void testHllOptionsErrors() {
    String[] baseParams = new String[] {"q", "*:*", "stats", "true", "indent", "true", "rows", "0"};
    SolrCore core = h.getCore();
    SchemaField foo_s = core.getLatestSchema().getField("foo_s");
    SchemaField foo_i = core.getLatestSchema().getField("foo_i");

    ignoreException("hllPreHashed");
    for (SchemaField field : new SchemaField[] {foo_s, foo_i}) {
      // whitebox - field
      SolrException ex =
          expectThrows(
              SolrException.class,
              () -> {
                HllOptions.parseHllOptions(
                    params("cardinality", "true", "hllPreHashed", "true"), field);
              });
      assertTrue(
          "MSG: " + ex.getMessage(),
          ex.getMessage().contains("hllPreHashed is only supported with Long"));
      // blackbox - field
      assertQEx(
          "hllPreHashed " + field.getName(),
          "hllPreHashed is only supported with Long",
          req(
              params("stats.field", "{!cardinality=true hllPreHashed=true}" + field.getName()),
              baseParams),
          ErrorCode.BAD_REQUEST);
    }

    // whitebox - function
    SolrException ex =
        expectThrows(
            SolrException.class,
            () -> {
              HllOptions.parseHllOptions(
                  params("cardinality", "true", "hllPreHashed", "true"), null);
            });
    assertTrue(
        "MSG: " + ex.getMessage(),
        ex.getMessage().contains("hllPreHashed is only supported with Long"));

    // blackbox - function
    assertQEx(
        "hllPreHashed function",
        "hllPreHashed is only supported with Long",
        req(
            params("stats.field", "{!func cardinality=true hllPreHashed=true}sum(foo_i,foo_l)"),
            baseParams),
        ErrorCode.BAD_REQUEST);

    ignoreException("accuracy");
    for (String invalid : new String[] {"-1", "1.1", "100"}) {
      // whitebox
      ex =
          expectThrows(
              SolrException.class,
              () -> {
                HllOptions.parseHllOptions(params("cardinality", invalid), foo_s);
              });
      assertTrue("MSG: " + ex.getMessage(), ex.getMessage().contains("number between 0 and 1"));
      // blackbox
      assertQEx(
          "cardinality=" + invalid,
          "number between 0 and 1",
          req(params("stats.field", "{!cardinality=" + invalid + "}foo_s"), baseParams),
          ErrorCode.BAD_REQUEST);
    }

    ignoreException("hllLog2m must be");
    for (int invalid : new int[] {HLL.MINIMUM_LOG2M_PARAM - 1, HLL.MAXIMUM_LOG2M_PARAM + 11}) {
      // whitebox
      ex =
          expectThrows(
              SolrException.class,
              () -> {
                HllOptions.parseHllOptions(
                    params("cardinality", "true", "hllLog2m", "" + invalid), foo_s);
              });
      assertTrue("MSG: " + ex.getMessage(), ex.getMessage().contains("hllLog2m must be"));

      // blackbox
      assertQEx(
          "hllLog2m=" + invalid,
          "hllLog2m must be",
          req(
              params("stats.field", "{!cardinality=true hllLog2m=" + invalid + "}foo_s"),
              baseParams),
          ErrorCode.BAD_REQUEST);
    }

    ignoreException("hllRegwidth must be");
    for (int invalid : new int[] {HLL.MINIMUM_REGWIDTH_PARAM - 1, HLL.MAXIMUM_REGWIDTH_PARAM + 1}) {
      // whitebox
      ex =
          expectThrows(
              SolrException.class,
              () -> {
                HllOptions.parseHllOptions(
                    params("cardinality", "true", "hllRegwidth", "" + invalid), foo_s);
              });
      assertTrue("MSG: " + ex.getMessage(), ex.getMessage().contains("hllRegwidth must be"));

      // blackbox
      assertQEx(
          "hllRegwidth=" + invalid,
          "hllRegwidth must be",
          req(
              params("stats.field", "{!cardinality=true hllRegwidth=" + invalid + "}foo_s"),
              baseParams),
          ErrorCode.BAD_REQUEST);
    }
  }

  // simple percentiles test
  public void testPercentiles() throws Exception {

    // NOTE: deliberately not in numeric order
    String percentiles = "10.0,99.9,1.0,2.0,20.0,30.0,40.0,50.0,60.0,70.0,80.0,98.0,99.0";
    List<String> percentilesList = StrUtils.splitSmart(percentiles, ',');

    // test empty case
    try (SolrQueryRequest query =
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.field",
            "{!percentiles='" + percentiles + "'}stat_f")) {
      SolrQueryResponse rsp = h.queryAndResponse(null, query);
      NamedList<Double> pout = extractPercentiles(rsp, "stat_f");

      assertEquals(percentilesList.size(), pout.size());
      Iterator<String> percentileIter = percentilesList.iterator();
      pout.forEach(
          (pKey, pVal) -> {
            String p = percentileIter.next();
            // ensure exact order, but all values should be null (empty result set)
            assertEquals(p, pKey);
            assertNull(pVal);
          });
    }

    int id = 0;
    // add trivial docs to test basic percentiles
    for (int i = 0; i < 100; i++) {
      // add the same values multiple times (diff docs)
      for (int j = 0; j < 5; j++) {
        assertU(adoc("id", ++id + "", "stat_f", "" + i));
      }
    }

    assertU(commit());

    try (SolrQueryRequest query =
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.field",
            "{!percentiles='" + percentiles + "'}stat_f")) {
      SolrQueryResponse rsp = h.queryAndResponse(null, query);
      NamedList<Double> pout = extractPercentiles(rsp, "stat_f");

      assertEquals(percentilesList.size(), pout.size());
      Iterator<String> percentileIter = percentilesList.iterator();
      pout.forEach(
          (pKey, pVal) -> {
            String p = percentileIter.next();
            assertEquals(p, pKey);
            assertEquals(Double.parseDouble(p), pVal, 1.0D);
          });
    }

    // test request for no percentiles
    try (SolrQueryRequest query =
        req("q", "*:*", "stats", "true", "stats.field", "{!percentiles=''}stat_f")) {
      SolrQueryResponse rsp = h.queryAndResponse(null, query);
      NamedList<Double> pout = extractPercentiles(rsp, "stat_f");
      assertNull(pout);
    }

    // non-numeric types don't support percentiles
    assertU(adoc("id", ++id + "", "stat_dt", "1999-05-03T04:55:01Z"));
    assertU(adoc("id", ++id + "", "stat_s", "cow"));

    assertU(commit());

    try (SolrQueryRequest query =
        req(
            "q",
            "*:*",
            "stats",
            "true",
            "stats.field",
            "{!percentiles='" + percentiles + "'}stat_dt",
            "stats.field",
            "{!percentiles='" + percentiles + "'}stat_s")) {
      SolrQueryResponse rsp = h.queryAndResponse(null, query);
      assertNull(extractPercentiles(rsp, "stat_dt"));
      assertNull(extractPercentiles(rsp, "stat_s"));
    }
  }

  @SuppressWarnings({"unchecked"})
  private NamedList<Double> extractPercentiles(SolrQueryResponse rsp, String key) {
    return ((NamedList<NamedList<NamedList<NamedList<Double>>>>) rsp.getValues().get("stats"))
        .get("stats_fields")
        .get(key)
        .get("percentiles");
  }

  /**
   * given a comboSize and an EnumSet of Stats, generates iterators that produce every possible enum
   * combination of that size
   */
  public static final class StatSetCombinations implements Iterable<EnumSet<Stat>> {
    // we need an array so we can do fixed index offset lookups
    private final Stat[] all;
    private final Combinations intCombos;

    public StatSetCombinations(int comboSize, EnumSet<Stat> universe) {
      // NOTE: should not need to sort, EnumSet uses natural ordering
      all = universe.toArray(new Stat[0]);
      intCombos = new Combinations(all.length, comboSize);
    }

    @Override
    public Iterator<EnumSet<Stat>> iterator() {
      return new Iterator<EnumSet<Stat>>() {
        final Iterator<int[]> wrapped = intCombos.iterator();

        @Override
        public void remove() {
          wrapped.remove();
        }

        @Override
        public boolean hasNext() {
          return wrapped.hasNext();
        }

        @Override
        public EnumSet<Stat> next() {
          EnumSet<Stat> result = EnumSet.noneOf(Stat.class);
          int[] indexes = wrapped.next();
          for (int index : indexes) {
            result.add(all[index]);
          }
          return result;
        }
      };
    }
  }
}
